"""
检查因子
1.根据因子路径;罗列因子名称
bydate:list  file
2.因子格式
对最新的,:1.时间-股票-值,2.时间-股票,kv  3.时间 v,4.时间,kv
3.根据不同类型的因子执行不同检查
4.预设问题,问题检查是否有问题
"""
import datetime
import os
from jili.core import load
from jili.data.db import get_calender,getdb_client,tradecal
from jili.tool.convert import str2datetime
from research.factor import factordata
class factor_check_tool:
    def __init__(self,factor_url):
        self.factor_url=factor_url
    def get_fnames(self,factor_url=None):
        if factor_url is None:
            factor_url=self.factor_url
        l=os.listdir(os.path.join(factor_url,"bydate"))
        ll=[]
        for i in l:
            url0=os.path.join(factor_url,"bydate",i)
            if os.path.isdir(url0):
                ll.append(i)

        return ll
    def check_model(self,factor_url=None,today=None):
        if today is None:
            now=datetime.datetime.now()
            today=datetime.datetime(now.year,now.month,now.day)
            if now.hour<=18:
                today=tradecal.getpretradeday(today)
        else:
            today=str2datetime(today)
        if factor_url is None:
            factor_url=self.factor_url
        f=factordata(url=factor_url)
        l=self.get_fnames(factor_url)
        rst={}
        for i in l:
            d0=f.get_maxtimekey(i)
            data=f.get_bydate(d0)
            if i[:3]=="is_":
                t="d-isin"
            elif isinstance(data,dict):
                l=list(data.keys())
                v=list(data.values())
                if isinstance(v,dict):
                    t="d-obj-k-v"
                else:
                    t="d-k-v"
            else:
                t="d-v"
            rst[i]=(d0,t,len(data),data)
            if d0==today:
                print("sys","正确",i,(d0,t,len(data)))
            else:
                print("sys","错误:不是最新",i, (d0, t, len(data)))
        return rst
    """
    1.是否缺失 d
    2.是否参数够
    3.值None个数
    """
    def check_all(self,factor_url=None,mongo_args=None,today=None,fnames=None,printflag="all",checklast=True):
        if factor_url is None:
            factor_url=self.factor_url
        if today is None:
            now=datetime.datetime.now()
            today=datetime.datetime(now.year,now.month,now.day)
            if now.hour<=18:
                today=tradecal.getpretradeday(today)
        else:
            today=str2datetime(today)
        if fnames is None:
            l=self.get_fnames(factor_url)
        else:
            l=fnames
        rst=[]
        for i in l:
            f = factordata(url=factor_url,name=i)
            url0=os.path.join(factor_url,"bydate",i)
            ll=os.listdir(url0)
            ll.sort()
            ii=ll[0]
            url1 = os.path.join(url0, ii)
            data = load(url1)
            d0=list(data.keys())[0]
            ii = ll[-1]
            url1 = os.path.join(url0, ii)
            data = load(url1)
            d1 = list(data.keys())[-1]
            v=data[d1]
            if today!=d1:
                today_gap=tradecal.count_tradedays(d1,today)
            else:
                today_gap=0
            info={"factor_name":i,"factor_url":factor_url,"start":d0,"end":d1,"data_null":[],"data_lt":{},"today_gap":today_gap}
            pren = None
            if i[:3] == "is_":
                info["data_model"]="d-isin"
                ds=None
                if (checklast and today_gap==0):
                    pred=tradecal.getpretradeday(today)
                    ds=get_calender(pred,today)
                elif not checklast :
                    ds=get_calender(d0,today)
                if ds is not None:
                    for d in ds:
                        data=f.get_bydate(d)
                        if data:
                            n=len(data)
                            if pren is not None:
                                if n<pren*0.9:
                                    info["data_lt"][str(d)[:10]]=[pren,n]
                                else:
                                    pren = n
                            else:
                                pren = n
                        else:
                            info["data_null"].append(str(d)[:10])
            elif isinstance(v,dict):
                l = list(v.keys())
                vv = list(v.values())
                if isinstance(vv, dict):
                    info["data_model"]= "d-obj-k-v"
                    ds = None
                    if (checklast and today_gap == 0):
                        pred = tradecal.getpretradeday(today)
                        ds = get_calender(pred, today)
                    elif not checklast:
                        ds = get_calender(d0, today)
                    if ds is not None:
                        for d in ds:
                            data = f.get_bydate(d)
                            if data:
                                n = len(data)
                                if pren is not None:
                                    if n<pren*0.9:
                                        info["data_lt"][str(d)[:10]] = [pren, n]
                                    else:
                                        pren = n
                                else:
                                    pren = n
                            else:
                                info["data_null"].append(str(d)[:10])
                else:
                    info["data_model"]= "d-k-v"
                    ds = None
                    if (checklast and today_gap == 0):
                        pred = tradecal.getpretradeday(today)
                        ds = get_calender(pred, today)
                    elif not checklast:
                        ds = get_calender(d0, today)
                    if ds is not None:
                        for d in ds:
                            data = f.get_bydate(d)
                            if data:
                                n = len(data)
                                if pren is not None:
                                    if n<pren*0.9:
                                        info["data_lt"][str(d)[:10]] = [pren, n]
                                    else:
                                        pren = n
                                else:
                                    pren = n
                            else:
                                info["data_null"].append(str(d)[:10])
            else:
                info["data_model"]= "d-v"
                ds = None
                if (checklast and today_gap == 0):
                    pred = tradecal.getpretradeday(today)
                    ds = get_calender(pred, today)
                elif not checklast:
                    ds = get_calender(d0, today)
                if ds is not None:
                    for d in ds:
                        data = f.get_bydate(d)
                        if data is None:
                            info["data_null"].append(str(d)[:10])
            rst.append(info)
            r1=len(info["data_null"])
            r2 = len(info["data_lt"])
            info["data_null_n"]=r1
            info["data_lt_n"] = r2
            if printflag=="all":
                if r1>0 and r2>0:
                    print("sys","缺失且数量异常:", info)
                elif r1>0:
                    print("sys","缺失:", info)
                elif r2>0:
                    print("sys","数量更新异常:", info)
                else:
                    print("sys","正常:", info)
            elif printflag=="null":
                if r1>0:
                    print("sys","缺失:", info)
            elif printflag=="lt":
                if r2 > 0:
                    print("sys","数量更新异常:", info)
            else:
                if r1>0 and r2>0:
                    print("sys","缺失且数量异常:", info)
                elif r1>0:
                    print("sys","缺失:", info)
                elif r2>0:
                    print("sys","数量更新异常:", info)


        if mongo_args is not None:
            db = getdb_client(dbname=mongo_args["mongdb_db"], ip=mongo_args["mongdb_ip"], auth=mongo_args["mongdb_auth"],
                              port=mongo_args["mongdb_port"])
            cu=db[mongo_args["mongdb_cu"]]
            cu.insert_many(rst)

        return rst



